const req = require('request');
const cheerio = require('cheerio');
var fs = require("fs");
var readline = require('readline');

const url = 'https://so.gushiwen.org/gushi/xiaoxue.aspx';
const tempfile = 'E:\\爬虫目录\\temp\\目录.txt'
var hastempfile = fs.existsSync(tempfile);
if(hastempfile){
  var data = fs.readFileSync(tempfile);
  if(data.length==0){
    get();
  }else{
    spliter(data);
  }
}else{
  get();
}

function get(){
  req.get({
    url : url,
    headers: {
      'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
      'referer': 'https://so.gushiwen.org/shiwen/'
    },
    encoding : 'utf-8'
  }, (err, res, body)=>{
    if(!err && res.statusCode == 200)
      fs.writeFileSync(tempfile, body);
      spliter(body);
  })
}

function spliter(html){
  const savefile = 'E:\\爬虫目录\\temp\\古诗\\目录.txt'
  var hastempfile = fs.existsSync(savefile);
  if(!hastempfile){
    let $ = cheerio.load(html);
    $('div.main3>div.right>div.sons>div.cont a').each(function(i,ele){
      let href=$(this).attr('href');
      if(href.indexOf('http')<0){
        href='https://so.gushiwen.org'+href;
      }
      let text=$(this).text();
      if(text.indexOf('更多')>-1) return true;
      fs.appendFileSync(savefile, href+' '+text+'\n','utf-8');
    })
  }else{
    spliter1();
  }
}

function spliter1(){
  const savefile = 'E:\\爬虫目录\\temp\\古诗\\目录.txt'
  var fRead = fs.createReadStream(savefile);
  var objReadline = readline.createInterface({
      input:fRead
  });
  objReadline.on('line',function (line) {
    let line_array = line.split(' ');
    let file_name = line_array[1];
    let son_url = line_array[0];
    get1(son_url,file_name);
  });
  objReadline.on('close',function () {

  }); 
}

function get1(url,name){
  req.get({
    url : url,
    headers: {
      'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
      'referer': 'https://so.gushiwen.org/shiwen/'
    },
    encoding : 'utf-8'
  }, (err, res, body)=>{
    if(!err && res.statusCode == 200)
      var tempfile1 = 'E:\\爬虫目录\\temp\\'+name+'.txt'
      fs.writeFileSync(tempfile1, body);
      spliter2(body,name);
  })
}

function spliter2(html,name){
    var savefile = 'E:\\爬虫目录\\temp\\古诗\\'+name+'.txt'
    let $ = cheerio.load(html);
    $('div.main3>div.left>div.sons>div.typecont>span>a').each(function(i,ele){
      let href=$(this).attr('href');
      if(href==null) return true;
      if(href.indexOf('http')<0){
        href='https://so.gushiwen.org'+href;
        fs.appendFileSync(savefile, href+'\n','utf-8');
      }
    })
}